﻿//
// LastFMPhotoCrawler.cs: Crawls Last.fm site for artist photos.
//
// Copyright 2010 Michael Goethe
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
//

using System;
using System.ComponentModel;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text.RegularExpressions;
using System.Threading;
using S9PalCommon;
using S9PalCommon.ArtistLibrary;

namespace LastFMPhotoCrawler
{
    /// <summary>
    /// Crawls Last.fm site for artist photos.
    /// </summary>
    public sealed class LastFMPhotoCrawler : Worker
    {   
        /// <summary>
        /// Stores relevant information about the photo being downloaded.
        /// </summary>
        struct PhotoBeingDownloadedToken
        {
            public ArtistPhoto artistPhoto;
            public string artistName;

            public PhotoBeingDownloadedToken(ArtistPhoto artistPhoto, string artistName)
            {
                this.artistPhoto = artistPhoto;
                this.artistName = artistName;
            }
        }

        #region Fields

        private CookieContainer cookieJar = new CookieContainer();    
        private Artist singleArtist;

        #endregion

        #region Constructors

        /// <summary>
        /// Creates a crawler which crawls for the entire library.
        /// </summary>
        public LastFMPhotoCrawler()
        {
            // Set properties that will identify the job.
            this.Icon = new Bitmap(System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream("LastFMPhotoCrawler.Resources.lastfm.png"));
            this.ItemCategory = "Artist";
            this.Name = "Last.fm Photo Crawler";

            // Workaround for periods in URIs being incorrectly handled.
            WorkaroundForSystemURIBug();
        }

        /// <summary>
        /// Creates a crawler which crawls for a single artist.
        /// </summary>
        /// <param name="artist">Artist to crawl for.</param>
        public LastFMPhotoCrawler(Artist artist)
        {
            // Set properties that will identify the job.
            this.ItemCategory = "Photo";
            this.Name = "Last.fm Photo Crawler (\"" + artist.Name + "\")";
            this.singleArtist = artist;

            // Workaround for periods in URIs being incorrectly handled.
            WorkaroundForSystemURIBug();
        }

        #endregion

        #region Protected Methods

        /// <summary>
        /// Activates the crawler. 
        /// </summary>
        protected override void OnStart()
        {

            if (singleArtist != null)
            {
                // Crawl for only the artist specified.
                DownloadPhotosForSingleArtist();
            }
            else
            {
                // Crawl for the entire library.
                if (S9PalCommon.Global.ArtistLibrary.Artists != null && S9PalCommon.Global.ArtistLibrary.Artists.Count > 0)
                    DownloadPhotosForEntireLibrary();
            }
        }

        #endregion

        #region Private Methods

        /// <summary>
        /// Crawl the entire library for photos.
        /// </summary>
        private void DownloadPhotosForEntireLibrary()
        {
            SortedList<String, Artist> artists = S9PalCommon.Global.ArtistLibrary.Artists;
            Artist resumeArtist = artists.First().Value;
            int resumeIndex;

            // Set ItemCount to the total number of artists in the library, so we have a denominator for our progress.
            ItemCount = S9PalCommon.Global.ArtistLibrary.Artists.Count;

            // If the crawler was paused, we want to pick back up where it left off. So set the artist to resume on.
            if (Global.Settings.ResumeArtist != null)
                resumeArtist = Global.Settings.ResumeArtist;

            // Get the initial index for the loop based upon the resume artist.
            resumeIndex = artists.IndexOfKey(resumeArtist.Name);
            if (resumeIndex < 0)
                resumeIndex = 0;

            // Loop through and crawl each artist.
            for (int i = resumeIndex; i < S9PalCommon.Global.ArtistLibrary.Artists.Count; i++)
            {
                Artist artist = artists.ElementAt(i).Value;

                // Update name and number to reflect the current item (used for output info for the job).
                CurrentItemName = artist.Name;
                CurrentItemNumber = i + 1;

                // Crawl for and download photos for the current artist.
                PullPhotosByArtist(artist, Global.Settings.PhotosToDownloadPerPass, Global.Settings.MaxPhotosPerArtist);

                // Save the library once we've finished pulling photos for the current artist.
                S9PalCommon.Global.ArtistLibrary.SaveLibrary();

                // If the user wants to pause or stop, do so.
                if (Status == WorkerStatus.Stopping)
                {
                    Status = WorkerStatus.Stopped;
                    return;
                }
                else if (Status == WorkerStatus.Pausing)
                {
                    // Set ResumeArtist so we can pick up where we left off later.
                    Global.Settings.ResumeArtist = artists.ElementAt(i).Value;
                    Global.Settings.SaveSettings();
                    Status = WorkerStatus.Paused;
                    return;
                }
            }

            Status = WorkerStatus.Finished;
        }

        /// <summary>
        /// Crawl for the specified artist.
        /// </summary>
        private void DownloadPhotosForSingleArtist()
        {
            // Set ItemCount to the target number of photos to download.
            ItemCount = Global.Settings.PhotosToDownloadPerPass;

            // Crawl for and download photos for the artist.
            PullPhotosByArtist(singleArtist, Global.Settings.PhotosToDownloadPerPass, Global.Settings.MaxPhotosPerArtist);

            // Stop if the user has requested to stop.
            if (Status == WorkerStatus.Stopping || Status == WorkerStatus.Pausing)
                Status = WorkerStatus.Stopped;
        }

        /// <summary>
        /// Get links to individual photo pages. These are relative URLs, as in they only account for the portion of the URL after the artist name.
        /// </summary>
        /// <param name="artist">Artist to get photo pages for.</param>
        /// <param name="numberOfNewPhotos">Target number of new photos to download.</param>
        /// <returns>List of relative photo page URLs.</returns>
        private List<String> GetPhotoPageLinks(Artist artist, int numberOfNewPhotos)
        {
            List<String> artistPhotoPageLinks = new List<String>();     // List of photo page URLs.
            Regex regex;                                                // Regular expression object.
            Match regexMatch;                                           // Regular expression match object.
            String html = null;                                         // HTML of the current page.                            
            bool backAtFirstPhoto = false;                              // Flags if we've run out of photos to download and have wound up back at the first photo.
            String firstPageLink = "";                                  // First photo page link we looked at. This is so we know when to stop.
            int pageNumber = 1;                                         // Page number iterator.

            // Crawl image library pages and grab links to individual image pages.
            // Each iteration of outer loop is one image library page, loop through as many as necessary to satisfy numberOfNewPhotos.
            while (artistPhotoPageLinks.Count < numberOfNewPhotos && !backAtFirstPhoto)
            {
                // If the user wants to stop or pause, then break out of the loop.
                if (Status == WorkerStatus.Stopping || Status == WorkerStatus.Pausing)
                    break;

                // Get the html of the current library page of photos.
                try
                {
                    html = GetHTMLFromURL("http://www.last.fm/music/" + SanitizeLastFMArtistName(artist.Name) + "/+images?page=" + pageNumber);
                }
                catch (WebException)
                {
                    // If no page for the artist was found, log and return null.
                    S9PalCommon.Global.Logger.Info("Could not locate Last.fm page for artist \"" + artist.Name + "\".");
                    return null;
                }

                // Set up the regular expression to match the signature of links to individual photo pages.
                // Only match the last portion of the URL after the artist name.
                regex = new Regex("(/\\+images/\\w*)\"\\s+class=\"pic\">");

                // Get the first match.
                regexMatch = regex.Match(html);

                // Loop through all the matches.
                while (regexMatch.Success && artistPhotoPageLinks.Count < numberOfNewPhotos)
                {
                    String lastFMImageID;
                    bool isAlreadyInLibrary = false;
                    String matchString;

                    // Get the first explicit group of the match, which is the photo link URL.
                    matchString = regexMatch.Groups[1].Value;

                    // Get the image ID from the URL. 
                    lastFMImageID = matchString.Substring(matchString.LastIndexOf("/") + 1);

                    // Check if we have come full-circle and are back at the original photo. This would arise if, say, we
                    // requested 10 pictures but there were only 5 uploaded for the artist.
                    if (matchString.CompareTo(firstPageLink) == 0)
                    {
                        backAtFirstPhoto = true;
                        break;
                    }

                    // Make sure we set the firstPageLink after comparing (above). 
                    // Otherwise, it will match on the first link without going through the list first. Oops.
                    if (String.IsNullOrEmpty(firstPageLink))
                        firstPageLink = matchString;

                    // Check if the current photo is already in the library.
                    foreach (ArtistPhoto artistPhoto in S9PalCommon.Global.ArtistLibrary.Artists[artist.Name].Photos)
                    {
                        if (lastFMImageID.CompareTo(artistPhoto.SourceFileID) == 0)
                        {
                            isAlreadyInLibrary = true;
                            break;
                        }
                    }

                    // If the photo is not in the library, add the photo page to the list of photo page URLs to download from.
                    if (!isAlreadyInLibrary)
                        artistPhotoPageLinks.Add(matchString);

                    // Set the regular expression match to the next match. This acts as a sort of iterator.
                    regexMatch = regexMatch.NextMatch();
                }

                // If the first page link is still empty after an initial pass then break out. 
                // This means that there is an artist page but no photos.
                if (String.IsNullOrEmpty(firstPageLink))
                {
                    S9PalCommon.Global.Logger.Info("Could not locate any photos for artist \"" + artist.Name + "\".");
                    break;
                }

                pageNumber++;
                Thread.Sleep(Global.Settings.HTTPSleepTime);
            }

            return artistPhotoPageLinks;
        }

        /// <summary>
        /// Crawl for and download photos for an artist.
        /// </summary>
        /// <param name="artist">Artist to crawl for.</param>
        /// <param name="numberOfNewPhotos">Target number of new photos to download.</param>
        /// <param name="maxNumberOfPhotos">Target maximum number of photos to allow in the library.</param>
        private void PullPhotosByArtist(Artist artist, int numberOfNewPhotos, int maxNumberOfPhotos)
        {
            List<String> artistPhotoPageLinks = new List<String>();

            // Adjust the number of new photos to download depending on whether we've hit the maximum.
            if (numberOfNewPhotos + artist.Photos.Count >= maxNumberOfPhotos)
                numberOfNewPhotos = maxNumberOfPhotos - artist.Photos.Count;
            
            // Get links to the individual photo pages.
            artistPhotoPageLinks = GetPhotoPageLinks(artist, numberOfNewPhotos);

            // Download photos from the photo pages.
            PullPhotosFromPageLinks(artist, artistPhotoPageLinks);
        }

        /// <summary>
        /// Crawl for and download photos from specified page links.
        /// </summary>
        /// <param name="artist">Artist that the photos are to be downloaded for.</param>
        /// <param name="artistPhotoPageLinks">List of relative photo page URLs.</param>
        private void PullPhotosFromPageLinks(Artist artist, List<String> artistPhotoPageLinks)
        {
            Regex regex;                    // Regular expression object.
            Match regexMatch;               // Regular expression match.
            String html;                    // HTML of the current individual photo page.
            String photoURL;                // Actual URL of the image file.
            String fileExtension;           // Image file's extension.
            WebClient webClient;            // WebClient object for downloading images.

            if (artistPhotoPageLinks == null || artist == null)
                return;

            // Get links to actual images and download them.
            for (int i = 0; i < artistPhotoPageLinks.Count; i++)
            {
                if (Status == WorkerStatus.Stopping || Status == WorkerStatus.Pausing) break;

                ArtistPhoto currentArtistPhoto = new ArtistPhoto();
                String lastFMImageID = artistPhotoPageLinks[i].Substring(artistPhotoPageLinks[i].LastIndexOf("/") + 1);
                
                // If we're in single artist mode, then increment the CurrentItemNumber for each photo.
                if (singleArtist != null)
                    CurrentItemNumber++;

                // Since there are other images on the page that have similar sources, first narrow down the section of the image we want.
                html = GetHTMLFromURL("http://www.last.fm/music/" + SanitizeLastFMArtistName(artist.Name) + artistPhotoPageLinks.ElementAt(i));
                regex = new Regex("<div class=\"the-image\">.*?</div>", RegexOptions.Singleline);
                regexMatch = regex.Match(html);

                // Now get the actual URL of the image.
                html = regexMatch.ToString();
                regex = new Regex("http://userserve-ak\\.last\\.fm/serve/.*\\.\\w*");
                regexMatch = regex.Match(html);

                // Set the URL to the match.
                photoURL = regexMatch.ToString();

                // Check if the extension is a jpg/jpeg. If so, download the image and use Last.fm's unique ID number for the filename.
                fileExtension = photoURL.Substring(photoURL.LastIndexOf("."));
                if (fileExtension.CompareTo(".jpg") == 0 || fileExtension.CompareTo(".jpeg") == 0)
                { 
                    String directoryPath = S9PalCommon.Global.ArtistLibrary.LibraryDirectory + "\\" + Helper.RemoveInvalidFileNameCharacters(artist.Name);
                    String filePath = directoryPath + "\\" + lastFMImageID + ".jpg";

                    currentArtistPhoto.SourceFileID = lastFMImageID;
                    currentArtistPhoto.LocalFileName = lastFMImageID + ".jpg";

                    // If the directory that we're saving to doesn't exist, create it.
                    if (!Directory.Exists(directoryPath))
                        Directory.CreateDirectory(directoryPath);

                    // Instantiate a new WebClient.
                    webClient = new WebClient();

                    // Download the file asynchronously.
                    webClient.DownloadFileAsync(new Uri(photoURL), filePath);
                }
                else if (fileExtension.CompareTo(".png") == 0 || fileExtension.CompareTo(".gif") == 0)
                {
                    // Since the file is not a JPEG, we do not want to do a simple download. Rather, we want to first download the
                    // image data and then convert and save it to a JPEG.
                    PhotoBeingDownloadedToken token;

                    currentArtistPhoto.SourceFileID = lastFMImageID;
                    currentArtistPhoto.LocalFileName = lastFMImageID + ".jpg";

                    // Instantiate a new token that will be passed along through the event handler.
                    token = new PhotoBeingDownloadedToken(currentArtistPhoto, artist.Name);

                    // Instantiate a new WebClient.
                    webClient = new WebClient();

                    // Attach an event handler, so we can run code when the data is finished downloading.
                    webClient.DownloadDataCompleted += new DownloadDataCompletedEventHandler(webClient_DownloadDataCompleted);

                    // Download the data asynchronously.
                    webClient.DownloadDataAsync(new Uri(photoURL), token);
                }

                Thread.Sleep(Global.Settings.HTTPSleepTime);
            }
        }

        /// <summary>
        /// Get HTML code at the specified URL.
        /// </summary>
        /// <param name="url">URL to get HTML from.</param>
        /// <returns>String of the page's HTML.</returns>
        private String GetHTMLFromURL(String url)
        {
            // Create a request at the specified URL.
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
            request.CookieContainer = cookieJar;

            // Get the request's response. Load the response into a stream, and then the stream into a reader.
            WebResponse response = request.GetResponse();
            Stream stream = response.GetResponseStream();
            StreamReader reader = new StreamReader(stream);

            // Get the HTML from the reader.
            string htmlText = reader.ReadToEnd();

            return htmlText;
        }

        /// <summary>
        /// Remove characters that don't play nice with Last.fm URIs.
        /// </summary>
        /// <param name="artistName">Artist name to sanitize.</param>
        /// <returns>String of the sanitized artist name.</returns>
        private String SanitizeLastFMArtistName(String artistName)
        {
            String sanitizedArtistName;

            // Replace bad characters with a plus, which serves as the primary conjunction operator on the Last.fm site.
            sanitizedArtistName = Regex.Replace(artistName, @"[\\/ ]", "+");

            return sanitizedArtistName;
        }

        /// <summary>
        /// Get encoder info from the specified MIME type.
        /// </summary>
        /// <param name="mimeType">MIME type to get the encoder from.</param>
        /// <returns>ImageCodecInfo of the matching codec. Returns null if no match was found.</returns>
        private ImageCodecInfo GetEncoderInfo(String mimeType)
        {
            // Get all available image encoders.
            ImageCodecInfo[] codecs = ImageCodecInfo.GetImageEncoders();

            // Find the correct codec.
            for (int i = 0; i < codecs.Length; i++)
                if (codecs[i].MimeType == mimeType)
                    return codecs[i];

            return null;
        }

        /// <summary>
        /// Saves an Image object to a JPEG file.
        /// </summary>
        /// <param name="image">Image to save.</param>
        /// <param name="path">Path to save the image at.</param>
        /// <param name="quality">Quality of the JPEG compression (1-100).</param>
        private void SaveImageAsJpeg(Image image, String path, int quality)
        {
            if (quality < 0 || quality > 100)
                throw new ArgumentOutOfRangeException("Quality value should be between 1 and 100.");

            EncoderParameter qualityParameter = new EncoderParameter(Encoder.Quality, quality);
            ImageCodecInfo jpegCodec = GetEncoderInfo("image/jpeg");
            EncoderParameters encoderParameters = new EncoderParameters(1);

            // Set first parameter to the quality parameter.
            encoderParameters.Param[0] = qualityParameter;

            // Save the image with the JPEG codec and quality parameter.
            image.Save(path, jpegCodec, encoderParameters);
        }

        /// <summary>
        /// Fixes issue where periods are inappropriately clipped from URIs.
        /// </summary>
        private void WorkaroundForSystemURIBug()
        {
            // There is a bug in System.Uri causing periods to be incorrectly stripped from URIs.
            // This is a workaround for said bug.
            MethodInfo getSyntax = typeof(UriParser).GetMethod("GetSyntax", System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic);
            FieldInfo flagsField = typeof(UriParser).GetField("m_Flags", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic);
            if (getSyntax != null && flagsField != null)
            {
                foreach (string scheme in new[] { "http", "https" })
                {
                    UriParser parser = (UriParser)getSyntax.Invoke(null, new object[] { scheme });
                    if (parser != null)
                    {
                        int flagsValue = (int)flagsField.GetValue(parser);
                        // Clear the CanonicalizeAsFilePath attribute
                        if ((flagsValue & 0x1000000) != 0)
                            flagsField.SetValue(parser, flagsValue & ~0x1000000);
                    }
                }
            }
        }

        #endregion

        #region Events

        private void webClient_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
        {
            PhotoBeingDownloadedToken token = (PhotoBeingDownloadedToken)e.UserState;
            String directoryPath = S9PalCommon.Global.ArtistLibrary.LibraryDirectory + "\\" + Helper.RemoveInvalidFileNameCharacters(token.artistName);
            String filePath = directoryPath + "\\" + token.artistPhoto.LocalFileName;
            MemoryStream imageStream = new MemoryStream(e.Result);
            Image image = new Bitmap(imageStream);

            // If the direcotry we're saving to doesn't exist, create it.
            if (!Directory.Exists(directoryPath))
                Directory.CreateDirectory(directoryPath);

            // Convert and save the image as a JPEG.
            SaveImageAsJpeg(image, filePath, Global.Settings.JPEGConversionQuality);
        }

        #endregion

    }
}
